# coding : utf-8

import scrapy
import bs4
from scrapy.contrib.spiders import CrawlSpider
from ..items import YigouItem


class YigouSpider(CrawlSpider):

    #
    # 易购房产网 规则
    #

    name = "yigouspider";
    allowed_domains = ['yigouf.com']
    start_urls = ['http://www.yigouf.com/index.php?m=content&c=index&a=lists&catid=9']

    def parse(self, response):
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')

        #
        # 城市列表
        #

        cityboxs = soup.select('.list_chs > ul ')
        for child in cityboxs:
            cols = child.find_all('a')
            for col in cols:
                city = col.get_text()
                if city != '不限' and city != '其他':
                    city_url = 'http://www.yigouf.com/' + col.get('href')
                    city_id = col.get('href').replace(
                            'index.php?m=content&c=index&a=lists&catid=9&salesid=&priceid=&areaid=', '').replace(
                            '&moldid=&o=listorder&s=b', '')
                    citys = {
                        'website': '易购网', 'web_url': 'yigouf.com',
                        'city': city, 'city_id': city_id
                    }
                    yield scrapy.Request(city_url,
                                         callback=self.parse_cityPage_url,
                                         meta=citys)

    def parse_cityPage_url(self, response):
        meta = response.meta
        data = response.body
        city = meta['city']
        city_id = meta['city_id']
        soup = bs4.BeautifulSoup(data, 'lxml')

        #
        # 城市 页面翻页
        #

        page_num = 1
        if city != '昌江县' and city != '琼中县' and city != '白沙县':
            page_num = int(soup.select('#pages > a ')[-2].get_text())

        for n in range(page_num):
            site_url = 'http://www.yigouf.com/index.php?m=content&c=index&a=lists&catid=9&salesid=&priceid=&areaid=' + city_id + '&moldid=&o=listorder&s=b&page=' + str(
                    n + 1)
            yield scrapy.Request(site_url,
                                 callback=self.parse_city_estate,
                                 meta=meta)

    def parse_city_estate(self, response):
        meta = response.meta
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')

        #
        # 获取楼盘信息
        #

        estates = soup.select('.list_main_li_h1  > b ')
        for child in estates:
            col = child.find_all('a')[0]
            estate = col.get_text()
            estate_url = col.get('href')
            estate_id = col.get('href').replace('http://www.yigouf.com/index.php?m=content&c=index&a=show&catid=9&id=',
                                                '')
            item = YigouItem()
            item['website'] = meta['website']
            item['web_url'] = meta['web_url']
            item['city'] = meta['city']
            item['area'] = ''
            item['city_id'] = meta['city_id']
            item['estate'] = estate
            item['estate_id'] = estate_id
            item['estate_url'] = estate_url
            yield item
